/** 
 * Project Name: dpinfrastructure 
 * File Name: PictureReptile.java 
 * Package Name: dpinfrastructure 
 * Date: 2017年2月2日上午10:41:20 
 * Copyright (c) 2017, shaohui_qdu@163.com All Rights Reserved. 
 * 
 */  
package dpinfrastructure;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;

import net.sh.dpinfrastructure.http.DPHttp;
import net.sh.dpinfrastructure.support.spring.configuration.DPHttpClientConfiguration;

/** 
 * 
 * 
 * @author shaohui 
 * @date: 2017年2月2日上午10:41:20 <br/> 
 * @version  
 * @since JDK 1.8
 */
public class PictureReptile {

    private final static Logger LOGGER = LoggerFactory.getLogger(PictureReptile.class);
    
    private static String domain = "http://zhaofuli.biz";
    
    private static Set<String> scannedLinks = new HashSet<>();
    public static Map<String, String> imgHeaders;
    
    static {
        imgHeaders = new HashMap<>();
        imgHeaders.put("Accept", "image/webp,image/*,*/*;q=0.8");
        imgHeaders.put("Accept-Encoding", "gzip, deflate, sdch");
        imgHeaders.put("Accept-Language", "zh-CN,zh;q=0.8");
        imgHeaders.put("Cache-Control", "no-cache");
        imgHeaders.put("Connection", "keep-alive");
        imgHeaders.put("Host", "images.zhaofulipic.com:8818");
        imgHeaders.put("Pragma", "no-cache");
        imgHeaders.put("referer", "http://zhaofuli.biz/");
        imgHeaders.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36");
    }
    
    public static void main(String[] args) throws IOException {
        AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(DPHttpClientConfiguration.class);
        try {
            DPHttp http = context.getBean("httpClient", DPHttp.class);
            scannLinks(http, domain);
        } finally {
            context.close();
        }
    }
    
    public static void downloadImg(DPHttp http, Map<String, String> imgHeaders, String uri, String imgName) throws IOException {
        byte[] bytes = http.get(uri, imgHeaders, null).getContent();
        File imgFile = new File("C:\\Users\\shaohui\\Desktop\\temp\\" + imgName);
        FileUtils.writeByteArrayToFile(imgFile , bytes, false);
    }
    
    public static void scannLinks(DPHttp http, String entry) throws IOException {
        Set<String> links = getAllLinks(http, entry);
        for (String link : links) {
            if (scannedLinks.contains(link)) {
                LOGGER.info("页面：" + link + "已经扫描过");
                continue;
            }
            scannedLinks.add(link);
            scannLinks(http, link);
        }
    }
    
    /**
     * 
     * 获取指定页面上所有的超链接
     * 
     * @param http
     * @param uri
     * @return
     *
     * @author shaohui
     * @throws IOException 
     * @date 2017年2月2日下午5:08:35
     * @since JDK 1.8
     */
    public static Set<String> getAllLinks(DPHttp http, String uri) throws IOException {
        LOGGER.info("扫描：" + uri);
        String pageHtml = http.get(uri).getContentAsString(Charset.forName("gbk"));
        Document document = Jsoup.parse(pageHtml);
        Elements links = document.select("a");
        Set<String> linkUris = new HashSet<>();
        for (Element element : links) {
            String href = element.attr("href");
            if (href.startsWith("http")) { // 其他域
                continue;
            }
            if (!href.endsWith(".html")) {
                continue;
            }
            if (href.startsWith("#")) { // 锚
                continue;
            }
            if (href.startsWith("/tags")) { // 标签
                continue;
            }
            if (href.contains("xiachedan")) {
                continue;
            }
            if (href.contains("xiurenwang")) {
                continue;
            }
            if (href.contains("youfanhao")) {
                continue;
            }
            if (href.contains("qiuchuchu")) {
                continue;
            }
            if (href.startsWith("/")) { // 相对于根目录
                linkUris.add(domain + href);
            } else { // 相对于当前目录
                linkUris.add(getParentUri(uri) + href);
            }
        }
        
        // 下载当前页上所有的图片
        Elements imgs = document.select("img");
        int imgCount = 0;
        for (Element element : imgs) {
            String src = element.attr("src");
            String imgName = src.substring(src.lastIndexOf("/") + 1, src.length());
            if (imgName.startsWith("1_") || imgName.endsWith(".gif") || imgName.endsWith(".png")
                    || imgName.endsWith("-L.jpg") || imgName.endsWith("-lp.jpg")) {
                continue;
            }
            try {
                Thread.sleep(200);
                downloadImg(http, imgHeaders, src, imgName);
                imgCount++;
            } catch (Exception e) {
                LOGGER.warn(src + "     下载失败");
            }
        }
        int linkSize = linkUris.size();
        if (linkSize == 0) {
            LOGGER.warn("又崩溃了");
            try {
                Thread.sleep(30000);
                return getAllLinks(http, uri);
            } catch (InterruptedException e) {
            }
        }
        LOGGER.info("共有有效连接" + linkUris.size() + "个，有效图片：" + imgCount + "个");
        return linkUris;
    }
 
    /**
     * 
     * 获取父目录 
     * 
     * @param uri
     * @return
     *
     * @author shaohui
     * @date 2017年2月2日下午5:21:20
     * @since JDK 1.8
     */
    public static String getParentUri(String uri) {
        return uri.substring(0, uri.lastIndexOf("/") + 1);
    }
}
